package com.chance.cc.crawler.development.scripts.du;

import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.chance.cc.crawler.core.CrawlerEnum;
import com.chance.cc.crawler.core.CrawlerRecordContext;
import com.chance.cc.crawler.core.downloader.HttpConstant;
import com.chance.cc.crawler.core.downloader.HttpPage;
import com.chance.cc.crawler.core.downloader.HttpRequest;
import com.chance.cc.crawler.core.downloader.HttpRequestBody;
import com.chance.cc.crawler.core.pipeline.result.CrawlerData;
import com.chance.cc.crawler.core.record.CrawlerRecord;
import com.chance.cc.crawler.core.record.CrawlerRequestRecord;
import com.chance.cc.crawler.core.script.crawler.CrawlerCommonScript;
import com.chance.cc.crawler.core.selector.Json;
import com.chance.cc.crawler.core.tags.CategoryTag;
import com.chance.cc.crawler.core.tags.KVTag;
import com.chance.cc.crawler.meta.core.bean.common.MetaResponse;
import com.chance.cc.crawler.meta.core.bean.crawler.CrawlerDomainKeys;
import com.chance.cc.crawler.meta.core.bean.crawler.CrawlerDomainUrls;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.time.DateFormatUtils;
import org.apache.commons.lang3.time.DateUtils;
import org.apache.http.NameValuePair;
import org.apache.http.client.utils.URLEncodedUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.FileInputStream;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.nio.charset.Charset;
import java.security.NoSuchAlgorithmException;
import java.text.ParseException;
import java.util.*;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit;
import java.util.regex.Pattern;

import static com.chance.cc.crawler.development.scripts.allfeild.AICCommonField.Tag_Field_Domain_Result_Json;
import static org.codehaus.groovy.runtime.EncodingGroovyMethods.md5;

/**
 * @author bx
 * @date 2020/12/6 0006 12:52
 */
public class DUProductCrawlerScript extends CrawlerCommonScript {

    private static Logger logger = LoggerFactory.getLogger(DUProductCrawlerScript.class);

    //start url
    public static final String startUrl = "api/v1/h5/commodity/fire/search/doCategoryDetail";
    public static final String startUrlFile = "api/v1/h5/commodity/fire/search/doCategoryDetail#file";

    //商品明细url
    public static final String productDetailUrl = "https://app.dewu.com/api/v1/h5/index/fire/flow/product/detail";
    public static final String sellProductlUrl = "https://app.dewu.com/api/v1/h5/inventory/price/h5/queryBuyNowInfo";
    public static final String askProductlUrl = "https://app.dewu.com/api/v1/h5/newbidding/buyer/view";

    //商品分类url 可以获取某类标签的所有分类
    public static final String categoryDetail = "https://app.dewu.com/api/v1/h5/commodity/fire/search/doCategoryDetail"; //分类url

    //列表页类型为brand
    public static final String seriesBrandUrl = "https://app.dewu.com/api/v1/h5/search/fire/commodity/detail_brand";

    //列表页类型为theme
    public static final String seriesThemeUrlRegular = "https://\\S*/api/v1/h5/commodity/fire/boutique-recommend/detail";
    public static final String seriesThemeUrl = "https://app.dewu.com/api/v1/h5/commodity/fire/boutique-recommend/detail";
    public static final String seriesThemePoizonUrl = "https://app.poizon.com/api/v1/h5/commodity/fire/boutique-recommend/detail";

    //列表页通用类型
    public static final String seriesCommonUrlRegular = "https://app.dewu.com/api/v1/h5/search/fire/search/list\\S*";
    public static final String seriesCommentUrlFormat = "https://app.dewu.com/api/v1/h5/search/fire/search/list?" +
            "title=%s&page=%s&sortType=%s&sortMode=1&limit=20&showHot=1&isAggr=1&catId=%s&unionId=%s";

    //列表页类型是3 -- 热销类型
    public static final String seriesDEWURegular = "https://\\S*/nezha-plus/detail/\\S*";
    public static final String seriesDEWURequestUrlRegular = "https://cdn.dewu.com/node-common/\\S*";
    public static final String seriesDEWURequestUrlFormat = "https://cdn.dewu.com/node-common/%s";

    //列表页类型是3 -- BoutiqueRecommend
    public static final String seriesBoutiqueRecommendRegular = "https://m.poizon.com/router/product/BoutiqueRecommendDetailPage\\S*";

    public static String sortType = "1"; //1 综合排序 3 新品排序

    public static final String site = "product";
    public static final String site_biz = "product";
    public static final String crawler_type = "crawler_type";
    public static final String tagBrandName = "brand_name";
    public static final String tagBrandId = "brand_id";
    public static final String tagRecommendId = "recommend_id";
    public static final String tagSpuIds = "spu_ids";
    public static final String tagProductId = "product_id";
    public static final String tagProductDetail = "product_detail";
    public static final String tagSellProduct = "sell_product";
    public static final String tagAskProduct = "ask_product";

    private static LinkedBlockingQueue<String> userAgents = new LinkedBlockingQueue<>();
    public static Object uaObject = new Object();

    private static List<String> imgs = new ArrayList<>();
    static {
        imgs.add("https://cdn.poizon.com/source-img/origin-img/20201225/e2da5ee2312b4030908b12070006f4ec.jpg");
        imgs.add("https://cdn.poizon.com/source-img/origin-img/20201225/2603140f61c34f43907690ed6e6dd9d6.jpg");
        imgs.add("https://cdn.poizon.com/source-img/origin-img/20201225/2aa72e8e738b4603a1cb7618c2ce7959.jpg");
        imgs.add("https://cdn.poizon.com/source-img/origin-img/20201225/4e9a700563d842448f7bfd31ba81d6d3.jpg");
    }

    @Override
    public List<CrawlerRequestRecord> parseLinks(CrawlerRequestRecord crawlerRecord, HttpPage page) {
        String requestUrl = crawlerRecord.getHttpRequest().getUrl();
        List<CrawlerRequestRecord> crawlerRequestRecords = new ArrayList<>();
//        logger.info("proxy note :{}",crawlerRecord.getHttpConfig().getProxy().getNote());

        if(crawlerRecord.isDownload() && !page.isDownloadSuccess()){
            logger.error("du product download url {} proxy has error ,will retry",requestUrl);
            addCrawlerRecords(crawlerRequestRecords,crawlerRecord);
            crawlerRecord.setNeedWashPage(false); //下载异常，不需要清洗页面
            return crawlerRequestRecords;
        }

        if(isUrlMatch(requestUrl,categoryDetail)){
            parseCategoryDetailLinks(crawlerRequestRecords,crawlerRecord,page);
        }else if(isUrlMatch(requestUrl,seriesBrandUrl)){
            parseSeriesBrandLinks(crawlerRequestRecords,crawlerRecord,page);
        }else if(isUrlMatch(requestUrl,seriesThemeUrlRegular)){
            parseSeriesThemeLinks(crawlerRequestRecords,crawlerRecord,page);
        }else if(isUrlMatch(requestUrl,seriesCommonUrlRegular)){
            parseSeriesCommonLinks(crawlerRequestRecords,crawlerRecord,page);
        }else if(isUrlMatch(requestUrl,seriesDEWURequestUrlRegular)){
            parseSeriesDEWULinks(crawlerRequestRecords,crawlerRecord,page);
        }else if(isUrlMatch(requestUrl,productDetailUrl)){
            parseProductDetailLinks(crawlerRequestRecords,crawlerRecord,page);
        }else if(isUrlMatch(requestUrl,sellProductlUrl)){
            parseSellProductLinks(crawlerRequestRecords,crawlerRecord,page);
        }else if(isUrlMatch(requestUrl,askProductlUrl)){
            parseAskProductLinks(crawlerRequestRecords,crawlerRecord,page);
        }

        return crawlerRequestRecords;
    }

    @Override
    public List<CrawlerData> washPage(CrawlerRequestRecord crawlerRecord, HttpPage page) {
        String requestUrl = crawlerRecord.getHttpRequest().getUrl();
        List<CrawlerData> crawlerDatas = new ArrayList<>();
        if (isUrlMatch(requestUrl,askProductlUrl) || isUrlMatch(requestUrl,productDetailUrl)){
            washProductInfo(crawlerDatas,crawlerRecord,page);
        }

        return crawlerDatas;
    }

    private void parseCategoryDetailLinks(List<CrawlerRequestRecord> crawlerRequestRecords,CrawlerRequestRecord crawlerRecord, HttpPage page){
        Json shoesJson = new Json(page.getRawText());
        try {
            List<String> shoesList = shoesJson.jsonPath($_type + ".data.list").all();
            for (String shoes : shoesList) {
                Json shoesCategory = new Json(shoes);
                String brandType = shoesCategory.jsonPath($_type + ".brand.type").get();
                if ("999".equals(brandType)){ //为你推荐
                    seriesRecommend(shoesCategory,crawlerRequestRecords,crawlerRecord);
                } else if ("0".equals(brandType)){
                    seriesNormal(shoesCategory,crawlerRequestRecords,crawlerRecord);
                } else {
                    logger.error("du product unknown brand type {}",brandType);
                }
            }
        } catch (Exception e) {
            logger.error("du product method parseCategoryDetailLinks has error {} ",e.getMessage());
            crawlerRecord.setNeedWashPage(false);
            addCrawlerRecords(crawlerRequestRecords,crawlerRecord);
        }
    }

    private void parseSeriesBrandLinks(List<CrawlerRequestRecord> crawlerRequestRecords,CrawlerRequestRecord crawlerRecord, HttpPage page){
        Json shoesJson = new Json(page.getRawText());
        try {
            List<String> shoesList = shoesJson.jsonPath($_type + ".data.itemList").all();
            String total = shoesJson.jsonPath($_type + ".data.total").get();
            for (String shoes : shoesList) {
                Json shoesDataJson = new Json(shoes);
                String spuId = shoesDataJson.jsonPath($_type + ".spuId").get();
                genProductDetailRecords(crawlerRequestRecords,crawlerRecord,spuId); //明细页
            }

            String brandId = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal(tagBrandId);
            if (Integer.valueOf(total) > 0){
                String lastId = shoesJson.jsonPath($_type + ".data.lastId").get();
                genSeriesBrandRecords(crawlerRequestRecords,crawlerRecord,brandId,lastId,Integer.parseInt(sortType)); //下一页
            }
        } catch (Exception e) {
            logger.error(e.getMessage());
            crawlerRecord.setNeedWashPage(false);
            addCrawlerRecords(crawlerRequestRecords,crawlerRecord);
        }
    }

    private void parseSeriesThemeLinks(List<CrawlerRequestRecord> crawlerRequestRecords,CrawlerRequestRecord crawlerRecord, HttpPage page){
        Json shoesJson = new Json(page.getRawText());
        try {
            List<String> shoesList = shoesJson.jsonPath($_type + ".data.list").all();
            String currentPage = shoesJson.jsonPath($_type + ".data.page").get();
            int nextPage = Integer.valueOf(currentPage) + 1;
            String recommendId = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal(tagRecommendId);
            String spuIds = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal(tagSpuIds);
            if (shoesList != null && shoesList.size() > 0){
                for (String shoes : shoesList) {
                    Json shoesDataJson = new Json(shoes);
                    String spuId = shoesDataJson.jsonPath($_type + ".id").get();
                    genProductDetailRecords(crawlerRequestRecords,crawlerRecord,spuId);
                }
                String requestUrl = page.getRequest().getUrl();
                genSeriesThemeRecords(crawlerRequestRecords,crawlerRecord,requestUrl,nextPage,Integer.parseInt(sortType),
                        recommendId,StringUtils.isNotBlank(spuIds)?Integer.valueOf(spuIds):null);//默认排序
            }

        } catch (Exception e) {
            logger.error(e.getMessage());
            crawlerRecord.setNeedWashPage(false);
            addCrawlerRecords(crawlerRequestRecords,crawlerRecord);
        }
    }

    private void parseSeriesCommonLinks(List<CrawlerRequestRecord> crawlerRequestRecords,CrawlerRequestRecord crawlerRecord, HttpPage page){
        Json shoesJson = new Json(page.getRawText());
        try {
            List<String> productList = shoesJson.jsonPath($_type + ".data.productList").all();
            String nextPage = shoesJson.jsonPath($_type + ".data.page").get();
            String total = shoesJson.jsonPath($_type + ".data.total").get();
            if (productList != null && productList.size() > 0){
                for (String product : productList) {
                    Json productJson = new Json(product);
                    String spuId = productJson.jsonPath($_type + ".spuId").get();
                    genProductDetailRecords(crawlerRequestRecords,crawlerRecord,spuId);
                }

                String brandId = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal(tagBrandId);
                if (Integer.valueOf(total) > 0 && Integer.valueOf(nextPage) > 0){
                    genSeriesCommonRecords(crawlerRequestRecords,crawlerRecord,"",nextPage,sortType,"1",brandId); //下一页
                }
            }

        } catch (Exception e) {
            logger.error(e.getMessage());
            crawlerRecord.setNeedWashPage(false);
            addCrawlerRecords(crawlerRequestRecords,crawlerRecord);
        }
    }

    private void parseSeriesDEWULinks(List<CrawlerRequestRecord> crawlerRequestRecords,CrawlerRequestRecord crawlerRecord, HttpPage page){
        Json shoesJson = new Json(page.getRawText());
        try {
            String products = shoesJson.jsonPath($_type + ".data").get();
            Json productsJson = new Json(products);
            List<String> componentList = productsJson.jsonPath($_type + ".componentList").all();
            for (String component : componentList) {
                Json componentJson = new Json(component);
                String name = componentJson.jsonPath($_type + ".name").get();
                if ("productFlowWithId".equals(name)){
                    String spuIds = componentJson.jsonPath($_type + ".props.ids").get();
                    String[] spuIdsArray = spuIds.split(",");
                    for (String spuId : spuIdsArray) {
                        genProductDetailRecords(crawlerRequestRecords,crawlerRecord,spuId);
                    }
                }

            }
        } catch (Exception e) {
            logger.error(e.getMessage());
            crawlerRecord.setNeedWashPage(false);
            addCrawlerRecords(crawlerRequestRecords,crawlerRecord);
        }
    }

    private void parseProductDetailLinks(List<CrawlerRequestRecord> crawlerRequestRecords,CrawlerRequestRecord crawlerRecord, HttpPage page){
        Json shoesJson = new Json(page.getRawText());
        try {
            String code = shoesJson.jsonPath($_type + ".code").get();
            String status = shoesJson.jsonPath($_type + ".status").get();
            if ("20801001".equals(code)){
                crawlerRecord.tagsCreator().bizTags().addCustomKV("is_del","delete");
                crawlerRecord.setNeedWashPage(true);
                return;
            }
            if (!"200".equals(code) || !"200".equals(status)){
                logger.error("du product detail download error,request url {},status code {},page rawtext{}"
                        ,page.getRequest().getUrl(),page.getStatusCode(),page.getRawText());
                throw new Exception("du product detail download error");
            }
            String spuId = shoesJson.jsonPath($_type + ".data.detail.spuId").get();
            List<String> all = shoesJson.jsonPath("data.image.spuImage.images[*].url").all();
            for (String img : all) {
                internalDownloadCommon(crawlerRequestRecords,crawlerRecord,img);
            }

            crawlerRecord.tagsCreator().bizTags().addCustomKV(tagProductDetail,page.getRawText());
            crawlerRecord.tagsCreator().bizTags().addCustomKV(tagProductId,spuId);
            if (!crawlerRecord.isNeedWashPage()){
                CrawlerRequestRecord crawlerRequestRecord = genSellProductRecords(crawlerRequestRecords,crawlerRecord,spuId);
            }
//            logger.info("success proxy note :{}",crawlerRecord.getHttpConfig().getProxy().getNote());
        } catch (Exception e) {
            logger.error(e.getMessage(),"will retry [{}]",crawlerRecord.getRecordKey());
            addCrawlerRecords(crawlerRequestRecords,crawlerRecord);
            crawlerRecord.getHttpRequest().addHeader("User-Agent",genRandomUA());
            crawlerRecord.setNeedWashPage(false);
        }
    }

    private void internalDownloadCommon(List<CrawlerRequestRecord> crawlerRequestRecords,CrawlerRequestRecord crawlerRecord,String url) throws NoSuchAlgorithmException {
        if (StringUtils.isNotBlank(url)){
            CrawlerRequestRecord commentCountRecord = CrawlerRequestRecord.builder()
                    .itemPageRequest(crawlerRecord)
                    .httpUrl(url)
                    .httpHead("appid","wxapp")
                    .httpHead("platform","h5")
                    .httpHead("content-type","application/json")
                    .releaseTime(System.currentTimeMillis())
                    .requestLabelTag(CrawlerEnum.CrawlerRequestType.internalDownload)
                    .build();
            commentCountRecord.getHttpConfig().setResponseTextGenerateHtml(false);
            crawlerRequestRecords.add(commentCountRecord);
        }
    }

    private void parseSellProductLinks(List<CrawlerRequestRecord> crawlerRequestRecords,CrawlerRequestRecord crawlerRecord, HttpPage page){
        Json shoesJson = new Json(page.getRawText());
        try {
            String code = shoesJson.jsonPath($_type + ".code").get();
            String status = shoesJson.jsonPath($_type + ".status").get();
            if ("20801001".equals(code)){
                crawlerRecord.tagsCreator().bizTags().addCustomKV("is_del","delete");
                crawlerRecord.setNeedWashPage(true);
                return;
            }
            if (!"200".equals(code) || !"200".equals(status)){
                logger.error("du sell product detail download error,request url {}",page.getRequest().getUrl());
                throw new Exception("du sell product detail download error");
            }
            String spuId = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal(tagProductId);
            for (String img : imgs) {
                internalDownloadCommon(crawlerRequestRecords,crawlerRecord,img);
            }
            CrawlerRequestRecord crawlerRequestRecord = genAskProductRecords(crawlerRequestRecords,crawlerRecord,spuId);
            crawlerRequestRecord.tagsCreator().bizTags().addCustomKV(tagSellProduct,page.getRawText());
        } catch (Exception e) {
            logger.error(e.getMessage(),"will retry [{}]",crawlerRecord.getRecordKey());
            addCrawlerRecords(crawlerRequestRecords,crawlerRecord);
            crawlerRecord.getHttpRequest().addHeader("User-Agent",genRandomUA());
            crawlerRecord.setNeedWashPage(false);
        }
    }

    private void parseAskProductLinks(List<CrawlerRequestRecord> crawlerRequestRecords,CrawlerRequestRecord crawlerRecord, HttpPage page){
        Json shoesJson = new Json(page.getRawText());
        try {
            String code = shoesJson.jsonPath($_type + ".code").get();
            String status = shoesJson.jsonPath($_type + ".status").get();
            if ("20801001".equals(code)){
                crawlerRecord.tagsCreator().bizTags().addCustomKV("is_del","delete");
                crawlerRecord.setNeedWashPage(true);
                return;
            }
            if (!"200".equals(code) || !"200".equals(status)){
                logger.error("du ask product detail download error,request url {}",page.getRequest().getUrl());
                throw new Exception("du ask product detail download error");
            }

            for (String img : imgs) {
                internalDownloadCommon(crawlerRequestRecords,crawlerRecord,img);
            }
        } catch (Exception e) {
            logger.error(e.getMessage(),"will retry [{}]",crawlerRecord.getRecordKey());
            addCrawlerRecords(crawlerRequestRecords,crawlerRecord);
            crawlerRecord.getHttpRequest().addHeader("User-Agent",genRandomUA());
            crawlerRecord.setNeedWashPage(false);
        }
    }


    private void seriesRecommend(Json shoesCategory,List<CrawlerRequestRecord> crawlerRequestRecords,CrawlerRequestRecord crawlerRecord){
        List<String> seriesList = shoesCategory.jsonPath($_type + ".seriesList").all();
        for (String series : seriesList) {
            Json seriesJson = new Json(series);
            String cn = seriesJson.jsonPath($_type + ".cn").get();
            String brandId = seriesJson.jsonPath($_type + ".brandId").get();
            String name = seriesJson.jsonPath($_type + ".name").get();
            CrawlerRequestRecord crawlerRequestRecord = null;
            if ("brand".equals(cn)){ // https://app.dewu.com/api/v1/h5/search/fire/commodity/detail_brand
                crawlerRequestRecord = genSeriesBrandRecords(crawlerRequestRecords, crawlerRecord, brandId, "0", Integer.parseInt(sortType));//默认排序
            } else if ("series".equals(cn)){
                crawlerRequestRecord = genSeriesCommonRecords(crawlerRequestRecords,crawlerRecord,"","0",sortType,"1",brandId);//默认排序
            } else if ("theme".equals(cn)){
                String routerUrl = seriesJson.jsonPath($_type + ".redirect.routerUrl").get();
                List<NameValuePair> parameters = URLEncodedUtils.parse(routerUrl.split("\\?")[1], Charset.forName("utf-8"));
                Integer spuIds = 0;
                Integer recommendId = 0;
                for (NameValuePair parameter : parameters) {
                    switch (parameter.getName()) {
                        case "spuIds":  spuIds = Integer.valueOf(parameter.getValue()); break;
                        case "recommendId":  recommendId = Integer.valueOf(parameter.getValue()); break;
                    }
                }
                String requestUrl=null;
                if (routerUrl.startsWith("https://m.dewu.com")){
                    requestUrl = seriesThemeUrl;
                } else if (routerUrl.startsWith("https://m.poizon.com/")){
                    requestUrl = seriesThemePoizonUrl;
                }
                crawlerRequestRecord = genSeriesThemeRecords(crawlerRequestRecords,crawlerRecord,requestUrl,1,Integer.parseInt(sortType),recommendId,spuIds);//默认排序
                crawlerRequestRecord.tagsCreator().bizTags().addCustomKV(tagRecommendId,recommendId);
                crawlerRequestRecord.tagsCreator().bizTags().addCustomKV(tagSpuIds,spuIds);
            } else {
                logger.error("du product brand {} unknown type {}",name,cn);
            }

            if (crawlerRequestRecord != null){
                crawlerRequestRecord.tagsCreator().bizTags().addCustomKV(tagBrandName,name);
                crawlerRequestRecord.tagsCreator().bizTags().addCustomKV(tagBrandId,brandId);
            }
        }
    }

    private void seriesNormal(Json shoesCategory,List<CrawlerRequestRecord> crawlerRequestRecords,CrawlerRequestRecord crawlerRecord) throws UnsupportedEncodingException {
        List<String> seriesList = shoesCategory.jsonPath($_type + ".seriesList").all();
        for (String series : seriesList) {
            Json seriesJson = new Json(series);
            String cn = seriesJson.jsonPath($_type + ".type").get();
            String name = seriesJson.jsonPath($_type + ".name").get();

            if ("1".equals(cn)){ // https://app.dewu.com/api/v1/h5/search/fire/commodity/detail_brand
                String brandId = seriesJson.jsonPath($_type + ".brandId").get();
                CrawlerRequestRecord crawlerRequestRecord = genSeriesCommonRecords(crawlerRequestRecords,crawlerRecord,
                        URLEncoder.encode(name,"utf-8"),"0",sortType,"0",brandId);//默认排序
                crawlerRequestRecord.tagsCreator().bizTags().addCustomKV(tagBrandName,name);
                crawlerRequestRecord.tagsCreator().bizTags().addCustomKV(tagBrandId,brandId);

            } else if ("2".equals(cn)){
                String brandId = seriesJson.jsonPath($_type + ".brandId").get();
                CrawlerRequestRecord crawlerRequestRecord = genSeriesCommonRecords(crawlerRequestRecords,crawlerRecord,
                        URLEncoder.encode(name,"utf-8"),"0",sortType,"1",brandId);//默认排序
                crawlerRequestRecord.tagsCreator().bizTags().addCustomKV(tagBrandName,name);
                crawlerRequestRecord.tagsCreator().bizTags().addCustomKV(tagBrandId,brandId);

            } else if ("3".equals(cn)){
                String redirectVal = seriesJson.jsonPath($_type + ".redirect.val").get();
                if(isUrlMatch(redirectVal,seriesDEWURegular)){ //热销类型
                    String[] vals = redirectVal.split("/");
                    String requestUrl = String.format(seriesDEWURequestUrlFormat, vals[vals.length - 1]);
                    CrawlerRequestRecord crawlerRequestRecord = genSeriesListRecords(crawlerRequestRecords,crawlerRecord,requestUrl);
                    crawlerRequestRecord.tagsCreator().bizTags().addCustomKV(tagBrandName,name);
                    crawlerRequestRecord.tagsCreator().bizTags().addCustomKV(tagBrandId,vals[vals.length - 1]);
                } else if (isUrlMatch(redirectVal,seriesBoutiqueRecommendRegular)){
                    List<NameValuePair> parameters = URLEncodedUtils.parse(redirectVal.split("\\?")[1], Charset.forName("utf-8"));
                    Integer recommendId = 0;
                    for (NameValuePair parameter : parameters) {
                        switch (parameter.getName()) {
                            case "recommendId":  recommendId = Integer.valueOf(parameter.getValue()); break;
                        }
                    }
                    String requestUrl=null;
                    if (redirectVal.startsWith("https://m.dewu.com")){
                        requestUrl = seriesThemeUrl;
                    } else if (redirectVal.startsWith("https://m.poizon.com/")){
                        requestUrl = seriesThemePoizonUrl;
                    }
                    CrawlerRequestRecord crawlerRequestRecord = genSeriesThemeRecords(crawlerRequestRecords,crawlerRecord,requestUrl,1,Integer.parseInt(sortType),"h5",recommendId,null);//默认排序
                    crawlerRequestRecord.tagsCreator().bizTags().addCustomKV(tagBrandName,name);
                    crawlerRequestRecord.tagsCreator().bizTags().addCustomKV(tagBrandId,recommendId);
                    crawlerRequestRecord.tagsCreator().bizTags().addCustomKV(tagRecommendId,recommendId);
                } else {
                    CrawlerRequestRecord crawlerRequestRecord = genSeriesCommonRecords(crawlerRequestRecords,crawlerRecord,URLEncoder.encode(name,"utf-8"),"0",sortType,"11",URLEncoder.encode(redirectVal,"utf-8"));//默认排序
                    crawlerRequestRecord.tagsCreator().bizTags().addCustomKV(tagBrandName,name);
                    crawlerRequestRecord.tagsCreator().bizTags().addCustomKV(tagBrandId,redirectVal);
                }

            } else {
                logger.error("du product brand {} unknown type {}",name,cn);
            }
        }
    }

    private CrawlerRequestRecord genSeriesBrandRecords(List<CrawlerRequestRecord> crawlerRequestRecords,CrawlerRequestRecord crawlerRecord,
                                       String brandId,String lastId,int sortType){

        CrawlerRequestRecord crawlerTurnPageRequestRecord = genSeriesListRecords(crawlerRequestRecords,crawlerRecord,seriesBrandUrl);
        crawlerTurnPageRequestRecord.setRecordKey(seriesBrandUrl+brandId+lastId+sortType);
        HttpRequest httpRequest = crawlerTurnPageRequestRecord.getHttpRequest();
        httpRequest.addHeader("content-type","application/json");
        httpRequest.setMethod("POST");
        Map<String,Object> productParams = new HashMap<>();
        productParams.put("brandId",brandId);
        productParams.put("lastId",lastId);
        productParams.put("sortType",sortType);
        productParams.put("sortMode",1);
        productParams.put("limit",20);
        productParams.put("categoryIds",new ArrayList<>());
        productParams.put("debugAgg",true);
        productParams.put("aggregation",true);
        String productSign = genSignature(productParams);
        productParams.put("sign",productSign);
        HttpRequestBody httpRequestBody = HttpRequestBody.json(JSON.toJSONString(productParams), "utf-8");
        httpRequest.setRequestBody(httpRequestBody);

        return crawlerTurnPageRequestRecord;
    }

    private CrawlerRequestRecord genSeriesThemeRecords(List<CrawlerRequestRecord> crawlerRequestRecords,CrawlerRequestRecord crawlerRecord,
                                                       String url,Integer pageNum,Integer sortType,Integer recommendId,Integer... spuIds){

        return genSeriesThemeRecords(crawlerRequestRecords, crawlerRecord,url, pageNum, sortType,"wxapp", recommendId, spuIds);
    }

    private CrawlerRequestRecord genSeriesThemeRecords(List<CrawlerRequestRecord> crawlerRequestRecords,CrawlerRequestRecord crawlerRecord,
                                       String url,Integer pageNum,Integer sortType,String appId,Integer recommendId,Integer... spuIds){

        CrawlerRequestRecord crawlerTurnPageRequestRecord = genSeriesListRecords(crawlerRequestRecords,crawlerRecord,url);
        crawlerTurnPageRequestRecord.setRecordKey(url+pageNum+sortType+appId+recommendId+
                (spuIds == null?"":StringUtils.joinWith("-",spuIds)));
        HttpRequest httpRequest = crawlerTurnPageRequestRecord.getHttpRequest();
        httpRequest.addHeader("content-type","application/json");
        httpRequest.addHeader("appid",appId);
        httpRequest.setMethod("POST");
        Map<String,Object> params = new HashMap<>();
        params.put("pageNum",pageNum);
        params.put("pageSize",10);
        params.put("spuIds",(spuIds == null? new ArrayList<>():Arrays.asList(spuIds)));
        params.put("recommendId",recommendId);

        Map<String,Object> aggregationMap = new LinkedHashMap<>();
        aggregationMap.put("aggregation",true);
        aggregationMap.put("sortType",sortType);
        aggregationMap.put("sortMode",1);

        Map<String,String> filtersMap = new HashMap<>();
        aggregationMap.put("filters",filtersMap);
        params.put("aggregation",aggregationMap);

        String sign= null;
        if (seriesThemeUrl.equals(url)){
            sign = genSignature(params);
        } else if (seriesThemePoizonUrl.equals(url)){
            sign = genSignature(params,"048a9c4943398714b356a696503d2d36");
        }
        params.put("sign",sign);
        HttpRequestBody httpRequestBody = HttpRequestBody.json(JSON.toJSONString(params), "utf-8");
        httpRequest.setRequestBody(httpRequestBody);

        return crawlerTurnPageRequestRecord;
    }

    private CrawlerRequestRecord genSeriesCommonRecords(List<CrawlerRequestRecord> crawlerRequestRecords,CrawlerRequestRecord crawlerRecord,
                                       String title,String page,String sortType,String catId,String unionId){
        String requestUrl = String.format(seriesCommentUrlFormat,title,page,sortType,catId,unionId);
        return genSeriesCommonRecords(crawlerRequestRecords, crawlerRecord,requestUrl);
    }

    private CrawlerRequestRecord genSeriesCommonRecords(List<CrawlerRequestRecord> crawlerRequestRecords,CrawlerRequestRecord crawlerRecord,String requestUrl){

        List<NameValuePair> parameters = URLEncodedUtils.parse(requestUrl.split("\\?")[1], Charset.forName("utf-8"));
        Map<String,Object> params = new HashMap<>();
        for (NameValuePair parameter : parameters) {
            params.put(parameter.getName(),parameter.getValue());
        }
        String sign = genSignature(params);
        requestUrl = requestUrl + "&sign="+sign;

        return genSeriesListRecords(crawlerRequestRecords,crawlerRecord,requestUrl);
    }

    //商品详情
    private void genProductDetailRecords(List<CrawlerRequestRecord> crawlerRequestRecords,CrawlerRequestRecord crawlerRecord,
                                         String spuId){
        String crawlerType = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal(crawler_type);
        boolean isWash = false;
        if ("findList".equals(crawlerType)) {
            isWash = true;
        } else if (StringUtils.isBlank(crawlerType)){
            logger.error("du product crawler need [crawler_type]");
            return;
        }

        CrawlerRequestRecord crawlerItemRequestRecord = CrawlerRequestRecord.builder()
                .itemPageRequest(crawlerRecord)
                .httpUrl(productDetailUrl)
                .recordKey(StringUtils.joinWith("-",domain(),site,CrawlerEnum.CrawlerDataType.article.enumVal(),spuId))
                .releaseTime(System.currentTimeMillis())
                .needWashed(isWash)
                .copyBizTags()
                .build();
        crawlerRequestRecords.add(crawlerItemRequestRecord);

        HttpRequest httpRequest = crawlerItemRequestRecord.getHttpRequest();
        httpRequest.setMethod("POST");
        Map<String,Object> params = new HashMap<>();
        params.put("spuId",spuId);
        params.put("productSourceName","");
        params.put("propertyValueId","0");

        String sign = genSignature(params);
        params.put("sign",sign);
        HttpRequestBody httpRequestBody = HttpRequestBody.json(JSON.toJSONString(params), "utf-8");
        httpRequest.setRequestBody(httpRequestBody);

        httpRequest.addHeader("Host","app.dewu.com");
        httpRequest.addHeader("Connection","keep-alive");
        httpRequest.addHeader("User-Agent",genRandomUA());
        httpRequest.addHeader("Content-Type","application/json");
        httpRequest.addHeader("platform","h5");
        httpRequest.addHeader("AppId","wxapp");
        httpRequest.addHeader("appVersion","4.4.0");
        httpRequest.addHeader("charset","utf-8");
        httpRequest.addHeader("Accept-Encoding","gzip,compress,br,deflate");
        httpRequest.addHeader("Referer","https://servicewechat.com/wx3c12cdd0ae8b1a7b/225/page-frame.html");
        httpRequest.addHeader("wxapp-login-token","");
    }

    private String genRandomUA(){
//        int part1 = RandomUtils.nextInt(1000, 9999);
//        int part2 = RandomUtils.nextInt(1000, 1999);
//        int part3 = RandomUtils.nextInt(100, 999);
//
//        return String.format("Mozilla/5.0 (Linux; Android 8.0.0; MI 5s Build/OPR1.170623.032; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/78.0.3904.62 XWEB/2797 MMWEBSDK/20210501 Mobile Safari/537.36 MMWEBID/%s MicroMessenger/8.0.6.%s(0x28000%s) Process/appbrand0 WeChat/arm64 Weixin NetType/WIFI Language/zh_CN ABI/arm64 MiniProgramEnv/android",
//                part1,part2,part3);
        String ua = null;
        try {
            ua = userAgents.poll(3, TimeUnit.SECONDS);
            logger.info("poll new ua:{}",ua);
            userAgents.add(ua);
        } catch (InterruptedException e) {
            logger.error(e.getMessage());
        }

        if (StringUtils.isBlank(ua)){
            logger.error("extract ua is null, return default ua");
            ua = "Mozilla/5.0 (Linux; Android 8.0.0; MI 5s Build/OPR1.170623.032; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/78.0.3904.62 XWEB/2885 MMWEBSDK/20210601 Mobile Safari/537.36 MMWEBID/6446 MicroMessenger/8.0.7.1920(0x28000753) Process/appbrand0 WeChat/arm64 Weixin NetType/WIFI Language/zh_CN ABI/arm64 MiniProgramEnv/android";
        }
        return ua;
    }

    //购买信息
    private CrawlerRequestRecord genSellProductRecords(List<CrawlerRequestRecord> crawlerRequestRecords,CrawlerRequestRecord crawlerRecord,
                                         String spuId){

        CrawlerRequestRecord crawlerItemRequestRecord = CrawlerRequestRecord.builder()
                .itemPageRequest(crawlerRecord)
                .httpUrl(sellProductlUrl)
                .recordKey(sellProductlUrl+spuId)
                .releaseTime(System.currentTimeMillis())
                .needWashed(false)
                .notFilterRecord()
                .copyBizTags()
                .build();
        crawlerRequestRecords.add(crawlerItemRequestRecord);

        HttpRequest httpRequest = crawlerItemRequestRecord.getHttpRequest();
        httpRequest.setMethod("POST");
        Map<String,Object> params = new HashMap<>();
        params.put("spuId",spuId);

        String sign = genSignature(params);
        params.put("sign",sign);
        HttpRequestBody httpRequestBody = HttpRequestBody.json(JSON.toJSONString(params), "utf-8");
        httpRequest.setRequestBody(httpRequestBody);

        httpRequest.addHeader("Host","app.dewu.com");
        httpRequest.addHeader("Connection","keep-alive");
        httpRequest.addHeader("User-Agent",genRandomUA());
        httpRequest.addHeader("Content-Type","application/json");
        httpRequest.addHeader("platform","h5");
        httpRequest.addHeader("AppId","wxapp");
        httpRequest.addHeader("appVersion","4.4.0");
        httpRequest.addHeader("charset","utf-8");
        httpRequest.addHeader("Accept-Encoding","gzip,compress,br,deflate");
        httpRequest.addHeader("Referer","https://servicewechat.com/wx3c12cdd0ae8b1a7b/225/page-frame.html");
        httpRequest.addHeader("wxapp-login-token","");

        return crawlerItemRequestRecord;
    }

    //求购信息
    private CrawlerRequestRecord genAskProductRecords(List<CrawlerRequestRecord> crawlerRequestRecords,CrawlerRequestRecord crawlerRecord,
                                                       String spuId){

        CrawlerRequestRecord crawlerItemRequestRecord = CrawlerRequestRecord.builder()
                .itemPageRequest(crawlerRecord)
                .httpUrl(askProductlUrl)
                .recordKey(askProductlUrl+spuId)
                .releaseTime(System.currentTimeMillis())
                .notFilterRecord()
                .copyBizTags()
                .build();
        crawlerRequestRecords.add(crawlerItemRequestRecord);

        HttpRequest httpRequest = crawlerItemRequestRecord.getHttpRequest();
        httpRequest.setMethod("POST");
        Map<String,Object> params = new HashMap<>();
        params.put("spuId",spuId);

        String sign = genSignature(params);
        params.put("sign",sign);
        HttpRequestBody httpRequestBody = HttpRequestBody.json(JSON.toJSONString(params), "utf-8");
        httpRequest.setRequestBody(httpRequestBody);

        httpRequest.addHeader("Host","app.dewu.com");
        httpRequest.addHeader("Connection","keep-alive");
        httpRequest.addHeader("User-Agent",genRandomUA());
        httpRequest.addHeader("Content-Type","application/json");
        httpRequest.addHeader("platform","h5");
        httpRequest.addHeader("AppId","wxapp");
        httpRequest.addHeader("appVersion","4.4.0");
        httpRequest.addHeader("charset","utf-8");
        httpRequest.addHeader("Accept-Encoding","gzip,compress,br,deflate");
        httpRequest.addHeader("Referer","https://servicewechat.com/wx3c12cdd0ae8b1a7b/225/page-frame.html");
        httpRequest.addHeader("wxapp-login-token","");

        return crawlerItemRequestRecord;
    }

    private CrawlerRequestRecord genSeriesListRecords(List<CrawlerRequestRecord> crawlerRequestRecords,CrawlerRequestRecord crawlerRecord,String requestUrl){

        CrawlerRequestRecord crawlerTurnPageRequestRecord = CrawlerRequestRecord.builder()
                .turnPageRequest(crawlerRecord)
                .httpUrl(requestUrl)
                .httpHead("appid","wxapp")
                .releaseTime(System.currentTimeMillis())
                .notFilterRecord()
                .copyBizTags()
                .build();
        crawlerRequestRecords.add(crawlerTurnPageRequestRecord);
        return crawlerTurnPageRequestRecord;
    }

    private void washProductInfo(List<CrawlerData> crawlerDatas,CrawlerRequestRecord crawlerRecord, HttpPage page){
        CategoryTag categoryTag = crawlerRecord.tagsCreator().bizTags().getCategoryTag();
        String spuId = categoryTag.getKVTagStrVal(tagProductId);
        String crawlerType = categoryTag.getKVTagStrVal(crawler_type);
        String productDetail = categoryTag.getKVTagStrVal(tagProductDetail);
        String sellProduct = categoryTag.getKVTagStrVal(tagSellProduct);
        categoryTag.getKvTags().remove(tagProductDetail);
        categoryTag.getKvTags().remove(tagSellProduct);
        String productUrl = "http://m.poizon.com/router/product/ProductDetail?spuId=" + spuId + "&sourceName=shareDetail";
        //根据详情结果判断商品是否已经下架
        Json shoesJson = new Json(productDetail);
        String code = shoesJson.jsonPath($_type + ".code").get();
        String msg = shoesJson.jsonPath($_type + ".msg").get();

        String cateId = categoryTag.getKVTagStrVal("cateId");
        String category = categoryTag.getKVTagStrVal("category");
        String brandName = categoryTag.getKVTagStrVal(tagBrandName);
        String brandId = categoryTag.getKVTagStrVal(tagBrandId);
        JSONObject result = new JSONObject();
        result.put("spuId",spuId);
        result.put("category",category);
        result.put("cateId",cateId);
        result.put("brandName",brandName);
        result.put("brandId",brandId);

        String isDel = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal("is_del");
        if (StringUtils.isNotBlank(isDel) && "delete".equals(isDel)){
            logger.error("du product detail is not exist");
            //输出到删除队列
            CrawlerData crawlerData = CrawlerData.builder()
                    .data(crawlerRecord,page)
                    .dataId(StringUtils.joinWith("-", domain(),site, CrawlerEnum.CrawlerDataType.article.enumVal(), spuId))
                    .url(productUrl)
                    .releaseTime(System.currentTimeMillis())
                    .content(result.toJSONString())
                    .requestLabelTag(CrawlerEnum.CrawlerRequestType.result)
                    .resultLabelTag(CrawlerEnum.CrawlerDataType.article)
                    .flowInPipelineTag("redis_del_result")
                    .flowInPipelineTag("file_del_result")
//                    .flowInPipelineTag("console")
                    .build();
            crawlerData.setFilterPipelineResult(true);
            crawlerDatas.add(crawlerData);
        }
        if ("200".equals(code) && "ok".equals(msg)){
            if ("findList".equals(crawlerType)){
                //仅通过列表同步更多的商品ID

                CrawlerData crawlerData = CrawlerData.builder()
                        .data(crawlerRecord,page)
                        .dataId(StringUtils.joinWith("-", domain(),site, CrawlerEnum.CrawlerDataType.article.enumVal(), spuId))
                        .url(productUrl)
                        .releaseTime(System.currentTimeMillis())
                        .content(result.toJSONString())
                        .requestLabelTag(CrawlerEnum.CrawlerRequestType.result)
                        .resultLabelTag(CrawlerEnum.CrawlerDataType.article)
                        .flowInPipelineTag("redis_add_result")
                        .flowInPipelineTag("file_add_result")
//                        .flowInPipelineTag("console")
                        .build();
                crawlerData.setFilterPipelineResult(true);
                crawlerDatas.add(crawlerData);
            }
            if ("productDetail".equals(crawlerType)){
                //详情数据落地
                CrawlerData crawlerData = CrawlerData.builder()
                        .data(crawlerRecord, page)
                        .dataId(StringUtils.joinWith("-", domain(),site, CrawlerEnum.CrawlerDataType.article.enumVal(), spuId))
                        .url(productUrl)
                        .releaseTime(System.currentTimeMillis())
                        .addContentKV(tagProductDetail,productDetail)
                        .addContentKV(tagSellProduct, sellProduct)
                        .addContentKV(tagAskProduct, page.getRawText())
                        .requestLabelTag(CrawlerEnum.CrawlerRequestType.result)
                        .resultLabelTag(CrawlerEnum.CrawlerDataType.article)
                        .flowInPipelineTag("kafka_data_result")
                        .flowInPipelineTag("file_data_result")
//                        .flowInPipelineTag("console")
                        .build();
                crawlerData.setFilterPipelineResult(true);
                crawlerData.tagsCreator().bizTags().addDomain(domain());
                crawlerData.tagsCreator().bizTags().addSite(site);
                crawlerData.tagsCreator().bizTags().addSiteBiz(site_biz);
                crawlerDatas.add(crawlerData);
            }
        }else {
            logger.error("du_app product detail has unknown return");
            logger.error(productDetail);
        }

    }

    @Override
    public String domain() {
        return "du_app";
    }

    @Override
    public void initUrlRegulars() {
        addUrlRegular(startUrl);
        addUrlRegular(startUrlFile);
        addUrlRegular(categoryDetail);
        addUrlRegular(seriesBrandUrl);
        addUrlRegular(seriesThemeUrlRegular);
        addUrlRegular(seriesCommonUrlRegular);
        addUrlRegular(seriesDEWURequestUrlRegular);
        addUrlRegular(productDetailUrl);
        addUrlRegular(sellProductlUrl);
        addUrlRegular(askProductlUrl);
    }

    @Override
    public boolean crawlerCheck(CrawlerRequestRecord crawlerRecord) {
        if ("https://m.dewu.com/sns/v1/content/posts-note-list"
                .equals(crawlerRecord.getHttpRequest().getUrl())){
            return false;
        }
        CategoryTag categoryTag = crawlerRecord.tagsCreator().bizTags().getCategoryTag();
        String crawlerSite = categoryTag.getKVTagStrVal("site");
        return crawlerSite.equalsIgnoreCase(site);
    }

    @Override
    public void afterExecute(CrawlerRecordContext context) {

    }


    @Override
    public List<CrawlerRecord> prepareRequest(CrawlerRequestRecord requestRecord, List<CrawlerRequestRecord> supportSourceRecords) {
        List<CrawlerRecord> allItemRecords = new ArrayList<>();

        if(userAgents.size() <1){
            synchronized (uaObject){
                if (userAgents.size() <1){
                    try {
                        userAgents.addAll(IOUtils.readLines(new FileInputStream("/data/chance_crawler_runner/data/ua/ua_du_app.txt"), "utf-8"));
                    } catch (IOException e) {
                        logger.error(e.getMessage());
                    }
                }
            }
        }

        HttpRequest httpRequest = requestRecord.getHttpRequest();
        if (isUrlMatch(httpRequest.getUrl(),startUrlFile)){
            initFileUrls(requestRecord,allItemRecords);
            return allItemRecords;
        }
        if (requestRecord.tagsCreator().bizTags().hasKVTag(Tag_Field_Domain_Result_Json)) {
            KVTag domainResultJson = requestRecord.tagsCreator().bizTags().getCategoryTag().getKVTag(Tag_Field_Domain_Result_Json);
            CrawlerDomainUrls crawlerDomainUrls = JSON.parseObject(String.valueOf(domainResultJson.getVal()), CrawlerDomainUrls.class);
            String url = crawlerDomainUrls.getUrl();
            JSONObject contentObj = JSONObject.parseObject(url);
            requestRecord.setNeedParsedPage(true);
            genInitProductDetailRecord(requestRecord, httpRequest, contentObj);
            logger.info("record: [{}] start",requestRecord.getRecordKey());
        }

        if (supportSourceRecords == null || supportSourceRecords.size() <1){
            return super.prepareRequest(requestRecord,supportSourceRecords);
        }

        CrawlerRequestRecord keywordSupportSourceRecord = null;
        for (CrawlerRequestRecord supportSourceRecord : supportSourceRecords) {
            if (supportSourceRecord.getHttpRequest().getUrl().endsWith("/v1/meta/"+domain()+"/keys?site="+site)){
                keywordSupportSourceRecord = supportSourceRecord;
                break;
            }
        }

        if (keywordSupportSourceRecord != null){

            initAllCrawlerRecordByKeyword(requestRecord,keywordSupportSourceRecord,allItemRecords);// cookies userAgents 初始完毕后，才能初始record

        } else {
            logger.error("du product start urls cant be empty!");
        }
        return allItemRecords;
    }

    private void genInitProductDetailRecord(CrawlerRequestRecord requestRecord, HttpRequest httpRequest, JSONObject contentObj) {
        String spuId = contentObj.getString("spuId");
        String cateId = contentObj.getString("cateId");
        String category = contentObj.getString("category");
        String brandId = contentObj.getString("brandId");
        String brandName = contentObj.getString("brandName");
        httpRequest.setMethod(HttpConstant.Method.POST);
        requestRecord.setRecordKey(StringUtils.joinWith("-",domain(),site, CrawlerEnum.CrawlerDataType.article.enumVal(),spuId));
        requestRecord.setReleaseTime(System.currentTimeMillis());
        Map<String,Object> params = new HashMap<>();
        params.put("spuId",spuId);
        params.put("productSourceName","");
        params.put("propertyValueId","0");

        String sign = genSignature(params);
        params.put("sign",sign);
        HttpRequestBody httpRequestBody = HttpRequestBody.json(JSON.toJSONString(params), "utf-8");
        httpRequest.setRequestBody(httpRequestBody);

        httpRequest.addHeader("Host","app.dewu.com");
        httpRequest.addHeader("Connection","keep-alive");
        httpRequest.addHeader("User-Agent",genRandomUA());
        httpRequest.addHeader("Content-Type","application/json");
        httpRequest.addHeader("platform","h5");
        httpRequest.addHeader("AppId","wxapp");
        httpRequest.addHeader("appVersion","4.4.0");
        httpRequest.addHeader("charset","utf-8");
        httpRequest.addHeader("Accept-Encoding","gzip,compress,br,deflate");
        httpRequest.addHeader("Referer","https://servicewechat.com/wx3c12cdd0ae8b1a7b/225/page-frame.html");
        httpRequest.addHeader("wxapp-login-token","");
        requestRecord.tagsCreator().bizTags().getCategoryTag().getKvTags().remove(Tag_Field_Domain_Result_Json); //移除
        requestRecord.tagsCreator().requestTags().removeRequestType(CrawlerEnum.CrawlerRequestType.filter);
        requestRecord.setRecordKey(StringUtils.joinWith("-",domain(),site,spuId));
        requestRecord.tagsCreator().bizTags().addCustomKV(crawler_type,"productDetail");
        requestRecord.tagsCreator().bizTags().addCustomKV(tagProductId,spuId);
        requestRecord.tagsCreator().bizTags().addCustomKV("cateId",cateId);
        requestRecord.tagsCreator().bizTags().addCustomKV("category",category);
        requestRecord.tagsCreator().bizTags().addCustomKV(tagBrandId,brandId);
        requestRecord.tagsCreator().bizTags().addCustomKV(tagBrandName,brandName);
    }

    private void initFileUrls(CrawlerRequestRecord requestRecord, List<CrawlerRecord> allItemRecords) {
        List<CrawlerRequestRecord> crawlerRequestRecords = new ArrayList<>();
        String filePath = requestRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal("filePath");
        requestRecord.tagsCreator().bizTags().getCategoryTag().getKvTags().remove("filePath");
        try {
            List<String> allSpuIds = IOUtils.readLines(new FileInputStream(filePath),"utf-8");
            for (String allSpuId : allSpuIds) {
                JSONObject jsonObject = JSONObject.parseObject(allSpuId);
                String spuId = jsonObject.getString("productId");
                String category = jsonObject.getString("Category");
                String brandName = jsonObject.getString("category_two");
                String cateId = "";
                switch (category){
                    case "潮鞋" : cateId = "3"; break;
                    case "潮服" : cateId = "6"; break;
                    default : cateId = "";
                }
                requestRecord.tagsCreator().bizTags().addCustomKV(tagProductId,spuId);
                requestRecord.tagsCreator().bizTags().addCustomKV("cateId",cateId);
                requestRecord.tagsCreator().bizTags().addCustomKV("category",category);
                requestRecord.tagsCreator().bizTags().addCustomKV(tagBrandId,"");
                requestRecord.tagsCreator().bizTags().addCustomKV(tagBrandName,brandName);
                genProductDetailRecords(crawlerRequestRecords,requestRecord,spuId);
            }
            allItemRecords.addAll(crawlerRequestRecords);

        } catch (IOException e) {
            logger.error(e.getMessage(),e);
        }


    }

    public static void main(String[] args) throws UnsupportedEncodingException {

    }


    public static String genSignature(Map<String,Object> params) {
        return genSignature(params,"19bc545a393a25177083d4a748807cc0");
    }

    public static String genSignature(Map<String,Object> params,String salt) {
        if(params == null){
            return null;
        }
        String sign = "";
        StringBuffer sb = new StringBuffer();
        try {
            // 1. 字典升序排序
            SortedMap<String,Object> sortedMap = new TreeMap<>(params);
            // 2. 拼按URL键值对
            Set<String> keySet = sortedMap.keySet();
            for(String key : keySet){
                //sign不参与算法
                if(key.equals("sign")){
                    continue;
                }

                Object obj =  sortedMap.get(key);
                String value = obj.toString() ;
                if (obj instanceof Map){
                    value = JSON.toJSONString(obj);
                }
                if (obj instanceof List){
                    List list = (List) obj;
                    value = "";
                    for (Object o : list) {
                        if (value.length() > 0){
                            value += ",";
                        }
                        value += JSON.toJSONString(o);
                    }
//                    value = ","+JSON.toJSONString(obj);
                }
//                if ("[]".equals(value)){
//                    value = "";
//                }
                sb.append(key).append(value);

            }
            String uriString = sb.toString() + salt;
            // 3. MD5运算得到请求签名
            sign = md5(uriString);
        } catch (Exception e) {
            e.printStackTrace();
        }
        return sign;
    }


    private void addCrawlerRecords(List<CrawlerRequestRecord> crawlerRequestRecords,CrawlerRequestRecord crawlerRecord){

//        int count = 1;
//        CrawlerBusinessTags crawlerBusinessTags = crawlerRecord.tagsCreator().bizTags();
//        if (crawlerBusinessTags.hasKVTag("download_retry_count")){
//            count = (int) crawlerBusinessTags.getCategoryTag().getKVTag("download_retry_count").getVal();
//            if (count >= 20){
//                logger.error("du product info download he number of retries exceeds the limit" +
//                        ",request url {},download detail {}",crawlerRecord.getHttpRequest().getUrl());
//                return;
//            }
//        }
//
//        count++;
//        crawlerBusinessTags.addCustomKV("download_retry_count",count);

        CrawlerRequestRecord crawlerRequestRecord = CrawlerRequestRecord.builder()
                .turnPageRequest(crawlerRecord)
                .httpRequest(crawlerRecord.getHttpRequest())
                .releaseTime(System.currentTimeMillis())
                .copyBizTags()
                .build();
        crawlerRequestRecord.tagsCreator().requestTags().removeRequestType(CrawlerEnum.CrawlerRequestType.filter);
        crawlerRequestRecord.setNeedWashPage(crawlerRecord.isNeedWashPage());
//        logger.error("record [{}] will to reparse",JSONObject.toJSONString(crawlerRequestRecord));
        crawlerRequestRecords.add(crawlerRequestRecord);
    }

    private void initAllCrawlerRecordByKeyword(CrawlerRequestRecord requestRecord,
                                               CrawlerRequestRecord supportSourceRecord,
                                               List<CrawlerRecord> allItemRecords){
        try {
            String crawlerSortType = requestRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal("crawler_sort_type");
            if (StringUtils.isNotBlank(crawlerSortType)){
                sortType = crawlerSortType;
                requestRecord.tagsCreator().bizTags().getCategoryTag().getKvTags().remove("crawler_sort_type");
            }
            HttpPage httpPage = supportSourceRecord.getInternalDownloadPage();
            MetaResponse metaResponse = JSON.parseObject(httpPage.getRawText(), MetaResponse.class);
            if (metaResponse.getStatus() == 0 && metaResponse.getContent() != null){
                List<String> contents = (List<String>) metaResponse.getContent();
                for (String content : contents) {
                    CrawlerDomainKeys crawlerDomainKeys = JSON.parseObject(content, CrawlerDomainKeys.class);
                    String keyword = crawlerDomainKeys.getKeyword();
                    CrawlerRequestRecord crawlerRequestRecord = CrawlerRequestRecord.builder()
                            .turnPageRequest(requestRecord)
                            .httpUrl(categoryDetail)
                            .recordKey(categoryDetail+keyword)
                            .httpHeads(requestRecord.getHttpRequest().getHeaders())
                            .releaseTime(System.currentTimeMillis())
                            .copyBizTags()
                            .notFilterRecord()
                            .build();

                    HttpRequest httpRequest = crawlerRequestRecord.getHttpRequest();
                    httpRequest.addHeader("content-type","application/json");
                    httpRequest.addHeader("appid","wxapp");
                    httpRequest.setMethod("POST");
                    Map<String,Object> params = new HashMap<>();
                    params.put("catId",Integer.valueOf(keyword));

                    String sign = genSignature(params);
                    params.put("sign",sign);
                    HttpRequestBody httpRequestBody = HttpRequestBody.json(JSON.toJSONString(params), "utf-8");
                    httpRequest.setRequestBody(httpRequestBody);
                    crawlerRequestRecord.tagsCreator().bizTags().addCustomKV("cateId",keyword);
                    String category = "";
                    switch (keyword){
                        //todo add more case to add category info
                        case "3" : category = "潮鞋"; break;
                        case "6" : category = "潮服"; break;
                        default : logger.error("unknown du category type");
                    }
                    crawlerRequestRecord.tagsCreator().bizTags().addCustomKV("category",category);

                    allItemRecords.add(crawlerRequestRecord);
                }
            }
        } catch (Exception e) {
            logger.error(e.getMessage(),e);
        }
    }

    public long dateToTimestamp(String dataStr) throws ParseException {
        String regEx="[^0-9]+"; //正则表达式，用于匹配非数字串，+号用于匹配出多个非数字串
        Pattern pattern = Pattern.compile(regEx);

        if (dataStr.equals("刚刚")){
            return System.currentTimeMillis();
        } else if (Pattern.matches("\\d*秒前",dataStr)){
            int number = Integer.valueOf(pattern.split(dataStr)[0]);
            return (System.currentTimeMillis() - number*1000L);
        } else if (Pattern.matches("\\d*分钟前",dataStr)){
            int number = Integer.valueOf(pattern.split(dataStr)[0]);
            return (System.currentTimeMillis() - number*60*1000L);
        } else if (Pattern.matches("\\d*小时前",dataStr)){
            int number = Integer.valueOf(pattern.split(dataStr)[0]);
            return (System.currentTimeMillis() - number*60*60*1000L);
        } else if (Pattern.matches("\\d*小时\\d*分钟前",dataStr)){
            int hourNumber = Integer.valueOf(pattern.split(dataStr)[0]);
            int secondNumber = Integer.valueOf(pattern.split(dataStr)[1]);
            long totalMillis = hourNumber*60*60*1000L + secondNumber*60*1000L;
            return (System.currentTimeMillis() - totalMillis);
        } else if (Pattern.matches("\\d*天前",dataStr)){
            int number = Integer.valueOf(pattern.split(dataStr)[0]);
            return (System.currentTimeMillis() - number*24*60*60*1000L);
        } else if (dataStr.startsWith("今天")){
            String currentTime = DateFormatUtils.format(System.currentTimeMillis(), TimeForamtEnum.format1.getFormat());
            dataStr = dataStr.replace("今天", currentTime);
        } else if (dataStr.startsWith("昨天")){
            String yesterdayTime = DateFormatUtils.format(System.currentTimeMillis() - 60*60*24*1000L, TimeForamtEnum.format1.getFormat());
            dataStr = dataStr.replace("昨天", yesterdayTime);
        } else if (dataStr.startsWith("前天")){
            String beforeYesterdayTime = DateFormatUtils.format(System.currentTimeMillis() - 2*60*60*24*1000L, TimeForamtEnum.format1.getFormat());
            dataStr = dataStr.replace("前天", beforeYesterdayTime);
        } else if (Pattern.matches("\\d*月\\d*[日]*", dataStr)){
            Calendar cal = Calendar.getInstance();
            dataStr = cal.get(Calendar.YEAR)+"年"+dataStr;
        } else if (Pattern.matches("\\d*-\\d*", dataStr)){
            Calendar cal = Calendar.getInstance();
            dataStr = cal.get(Calendar.YEAR)+"-"+dataStr;
        }
        return DateUtils.parseDateStrictly(dataStr, TimeForamtEnum.allFormats()).getTime();
    }

    public enum TimeForamtEnum {
        format1("yyyy年MM月dd日"),
        format1_1("yyyy年MM月dd日 HH:mm:ss"),
        format1_2("yyyy年MM月dd日 HH:mm"),
        format1_3("yyyy年MM月dd日HH:mm"),
        format1_4("yyyy年MM月dd日 HH点mm分"),
        format1_5("yyyy年MM月dd日 HH点mm分ss秒"),
        format1_6("yyyy年MM月dd日HH点mm分"),
        format1_7("yyyy年MM月dd日HH点mm分ss秒"),


        format4("yyyy-MM-dd HH:mm:ss"),
        format5("yyyy-MM-dd HH:mm"),
        format6("yyyy-MM-dd"),

        format7("yyyy/MM/dd HH:mm:ss"),
        format7_1("MM/dd/yyyy HH:mm:ss"),
        format8("yyyy/MM/dd HH:mm"),
        format9("yyyy/MM/dd"),

        format10("yyyy.MM.dd HH:mm:ss"),
        format11("yyyy.MM.dd HH:mm"),
        format12("yyyy.MM.dd"),

        format13("EEE MMM d HH:mm:ss +0800 yyyy");

        private String format;

        private TimeForamtEnum(String format) {
            this.format = format;
        }

        public static String[] allFormats() {
            TimeForamtEnum[] timeForamtEnums = TimeForamtEnum.values();
            String[] formats = new String[timeForamtEnums.length];
            for (int count = 0; count < timeForamtEnums.length; count++) {
                formats[count] = timeForamtEnums[count].format;
            }
            return formats;
        }

        public String getFormat() {
            return format;
        }
    }
}
